{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/08, 22:00:24, Iteration: 100, Train Loss: 2.0117e+02, c1: 5.2666e-01, c2: 1.1670e-03, d1: 1.3684e+00, d2: 0.0000e+00, s: 1.4224e-02, l1_reg: 3.1796e+02, l2_reg: 4.9489e+01\n",
      "2020/09/08, 22:00:24, Iteration: 200, Train Loss: 1.5806e+02, c1: 2.4796e-01, c2: 7.4188e-04, d1: 1.2585e+00, d2: 0.0000e+00, s: 2.0567e-02, l1_reg: 3.1709e+02, l2_reg: 4.9265e+01\n",
      "2020/09/08, 22:00:25, Iteration: 300, Train Loss: 1.3863e+02, c1: 6.8116e-02, c2: 1.7680e-03, d1: 1.1413e+00, d2: 0.0000e+00, s: 1.4140e-02, l1_reg: 3.1635e+02, l2_reg: 4.9104e+01\n",
      "2020/09/08, 22:00:26, Iteration: 400, Train Loss: 1.2114e+02, c1: 9.8794e-02, c2: 8.7313e-04, d1: 1.0253e+00, d2: 0.0000e+00, s: 2.4599e-02, l1_reg: 3.1581e+02, l2_reg: 4.8931e+01\n",
      "2020/09/08, 22:00:27, Iteration: 500, Train Loss: 1.0689e+02, c1: 1.0342e-01, c2: 5.2255e-04, d1: 9.1321e-01, d2: 0.0000e+00, s: 2.9437e-02, l1_reg: 3.1542e+02, l2_reg: 4.8837e+01\n",
      "2020/09/08, 22:00:28, Iteration: 600, Train Loss: 9.1930e+01, c1: 5.5705e-02, c2: 6.9536e-04, d1: 7.9406e-01, d2: 0.0000e+00, s: 3.4796e-02, l1_reg: 3.1511e+02, l2_reg: 4.8754e+01\n",
      "2020/09/08, 22:00:29, Iteration: 700, Train Loss: 8.6945e+01, c1: 6.4348e-02, c2: 1.3057e-03, d1: 6.7453e-01, d2: 0.0000e+00, s: 3.4524e-02, l1_reg: 3.1486e+02, l2_reg: 4.8689e+01\n",
      "2020/09/08, 22:00:29, Iteration: 800, Train Loss: 6.5343e+01, c1: 2.1135e-02, c2: 7.9534e-04, d1: 5.5276e-01, d2: 0.0000e+00, s: 4.5293e-02, l1_reg: 3.1462e+02, l2_reg: 4.8619e+01\n",
      "2020/09/08, 22:00:30, Iteration: 900, Train Loss: 6.5690e+01, c1: 1.0804e-01, c2: 5.7024e-04, d1: 4.9183e-01, d2: 0.0000e+00, s: 4.4547e-02, l1_reg: 3.1430e+02, l2_reg: 4.8553e+01\n",
      "2020/09/08, 22:00:31, Iteration: 1000, Train Loss: 5.7343e+01, c1: 3.3089e-02, c2: 5.8524e-04, d1: 4.8182e-01, d2: 0.0000e+00, s: 4.3268e-02, l1_reg: 3.1403e+02, l2_reg: 4.8475e+01\n",
      "2020/09/08, 22:00:32, Iteration: 1100, Train Loss: 5.7863e+01, c1: 5.3547e-02, c2: 5.1179e-04, d1: 4.7391e-01, d2: 0.0000e+00, s: 4.3114e-02, l1_reg: 3.1388e+02, l2_reg: 4.8465e+01\n",
      "2020/09/08, 22:00:33, Iteration: 1200, Train Loss: 5.7989e+01, c1: 6.7242e-02, c2: 4.5862e-04, d1: 4.6679e-01, d2: 0.0000e+00, s: 4.4365e-02, l1_reg: 3.1378e+02, l2_reg: 4.8474e+01\n",
      "2020/09/08, 22:00:33, Iteration: 1300, Train Loss: 5.3142e+01, c1: 2.2478e-02, c2: 4.8066e-04, d1: 4.6088e-01, d2: 0.0000e+00, s: 4.4813e-02, l1_reg: 3.1373e+02, l2_reg: 4.8501e+01\n",
      "2020/09/08, 22:00:34, Iteration: 1400, Train Loss: 5.5917e+01, c1: 4.8930e-02, c2: 5.4865e-04, d1: 4.5537e-01, d2: 0.0000e+00, s: 4.1197e-02, l1_reg: 3.1373e+02, l2_reg: 4.8543e+01\n",
      "2020/09/08, 22:00:35, Iteration: 1500, Train Loss: 5.1056e+01, c1: 4.9360e-02, c2: 9.6106e-05, d1: 4.5159e-01, d2: 0.0000e+00, s: 2.8852e-02, l1_reg: 3.1355e+02, l2_reg: 4.8395e+01\n",
      "2020/09/08, 22:00:36, Iteration: 1600, Train Loss: 5.2335e+01, c1: 3.2637e-02, c2: 4.3177e-04, d1: 4.4754e-01, d2: 0.0000e+00, s: 1.9627e-02, l1_reg: 3.1348e+02, l2_reg: 4.8315e+01\n",
      "2020/09/08, 22:00:37, Iteration: 1700, Train Loss: 4.8948e+01, c1: 3.0976e-02, c2: 1.4177e-04, d1: 4.4433e-01, d2: 0.0000e+00, s: 1.1601e-02, l1_reg: 3.1332e+02, l2_reg: 4.8175e+01\n",
      "2020/09/08, 22:00:38, Iteration: 1800, Train Loss: 5.6198e+01, c1: 4.7862e-02, c2: 7.4113e-04, d1: 4.4000e-01, d2: 0.0000e+00, s: 1.1802e-02, l1_reg: 3.1337e+02, l2_reg: 4.8202e+01\n",
      "2020/09/08, 22:00:38, Iteration: 1900, Train Loss: 4.9385e+01, c1: 1.2032e-02, c2: 4.6113e-04, d1: 4.3571e-01, d2: 0.0000e+00, s: 1.2246e-02, l1_reg: 3.1343e+02, l2_reg: 4.8194e+01\n",
      "2020/09/08, 22:00:39, Iteration: 2000, Train Loss: 5.5955e+01, c1: 1.0347e-01, c2: 2.8329e-04, d1: 4.2775e-01, d2: 0.0000e+00, s: 9.2203e-03, l1_reg: 3.1366e+02, l2_reg: 4.8234e+01\n",
      "2020/09/08, 22:00:40, Iteration: 2100, Train Loss: 5.1195e+01, c1: 2.9665e-02, c2: 5.7386e-04, d1: 4.2490e-01, d2: 0.0000e+00, s: 1.1144e-02, l1_reg: 3.1385e+02, l2_reg: 4.8385e+01\n",
      "2020/09/08, 22:00:41, Iteration: 2200, Train Loss: 5.2666e+01, c1: 2.8819e-02, c2: 8.1446e-04, d1: 4.1640e-01, d2: 0.0000e+00, s: 8.0383e-03, l1_reg: 3.1402e+02, l2_reg: 4.8407e+01\n",
      "2020/09/08, 22:00:42, Iteration: 2300, Train Loss: 4.5444e+01, c1: 3.1297e-02, c2: 1.2618e-04, d1: 4.1052e-01, d2: 0.0000e+00, s: 8.3293e-03, l1_reg: 3.1406e+02, l2_reg: 4.8378e+01\n",
      "2020/09/08, 22:00:42, Iteration: 2400, Train Loss: 4.7190e+01, c1: 3.6126e-02, c2: 3.4001e-04, d1: 4.0177e-01, d2: 0.0000e+00, s: 6.9109e-03, l1_reg: 3.1433e+02, l2_reg: 4.8476e+01\n",
      "2020/09/08, 22:00:43, Iteration: 2500, Train Loss: 4.9252e+01, c1: 3.8668e-02, c2: 5.9292e-04, d1: 3.9456e-01, d2: 0.0000e+00, s: 5.4193e-03, l1_reg: 3.1451e+02, l2_reg: 4.8539e+01\n",
      "2020/09/08, 22:00:44, Iteration: 2600, Train Loss: 5.2928e+01, c1: 5.0271e-02, c2: 9.0123e-04, d1: 3.8888e-01, d2: 0.0000e+00, s: 6.4341e-03, l1_reg: 3.1480e+02, l2_reg: 4.8724e+01\n",
      "2020/09/08, 22:00:45, Iteration: 2700, Train Loss: 4.7868e+01, c1: 1.4767e-02, c2: 8.8267e-04, d1: 3.7564e-01, d2: 0.0000e+00, s: 5.5280e-03, l1_reg: 3.1524e+02, l2_reg: 4.8952e+01\n",
      "2020/09/08, 22:00:46, Iteration: 2800, Train Loss: 4.5818e+01, c1: 2.8424e-02, c2: 6.7354e-04, d1: 3.6241e-01, d2: 0.0000e+00, s: 5.2676e-03, l1_reg: 3.1567e+02, l2_reg: 4.9204e+01\n",
      "2020/09/08, 22:00:47, Iteration: 2900, Train Loss: 4.2810e+01, c1: 3.1022e-02, c2: 5.1945e-04, d1: 3.4514e-01, d2: 0.0000e+00, s: 5.9318e-03, l1_reg: 3.1621e+02, l2_reg: 4.9509e+01\n",
      "2020/09/08, 22:00:48, Iteration: 3000, Train Loss: 4.0200e+01, c1: 1.3566e-02, c2: 7.4431e-04, d1: 3.1400e-01, d2: 0.0000e+00, s: 7.0340e-03, l1_reg: 3.1694e+02, l2_reg: 4.9937e+01\n",
      "2020/09/08, 22:00:49, Iteration: 3100, Train Loss: 4.4911e+01, c1: 9.6864e-02, c2: 5.2532e-04, d1: 2.9971e-01, d2: 0.0000e+00, s: 6.9870e-03, l1_reg: 3.1771e+02, l2_reg: 5.0487e+01\n",
      "2020/09/08, 22:00:49, Iteration: 3200, Train Loss: 4.9621e+01, c1: 1.6432e-01, c2: 9.0479e-04, d1: 2.4142e-01, d2: 0.0000e+00, s: 6.5304e-03, l1_reg: 3.1900e+02, l2_reg: 5.1272e+01\n",
      "2020/09/08, 22:00:50, Iteration: 3300, Train Loss: 2.9186e+01, c1: 4.4045e-02, c2: 8.8493e-04, d1: 1.5933e-01, d2: 0.0000e+00, s: 7.1470e-03, l1_reg: 3.2074e+02, l2_reg: 5.2304e+01\n",
      "2020/09/08, 22:00:51, Iteration: 3400, Train Loss: 2.6896e+01, c1: 8.7406e-02, c2: 5.5736e-04, d1: 1.2582e-01, d2: 0.0000e+00, s: 7.3405e-03, l1_reg: 3.2184e+02, l2_reg: 5.3098e+01\n",
      "2020/09/08, 22:00:52, Iteration: 3500, Train Loss: 1.7477e+01, c1: 2.1021e-02, c2: 3.9063e-04, d1: 1.1469e-01, d2: 0.0000e+00, s: 8.0215e-03, l1_reg: 3.2228e+02, l2_reg: 5.3535e+01\n",
      "2020/09/08, 22:00:53, Iteration: 3600, Train Loss: 1.8549e+01, c1: 1.9571e-02, c2: 5.6277e-04, d1: 1.0965e-01, d2: 0.0000e+00, s: 8.2793e-03, l1_reg: 3.2255e+02, l2_reg: 5.3857e+01\n",
      "2020/09/08, 22:00:54, Iteration: 3700, Train Loss: 1.9158e+01, c1: 3.5986e-02, c2: 5.0945e-04, d1: 1.0465e-01, d2: 0.0000e+00, s: 8.4836e-03, l1_reg: 3.2278e+02, l2_reg: 5.4164e+01\n",
      "2020/09/08, 22:00:54, Iteration: 3800, Train Loss: 1.7789e+01, c1: 3.4242e-02, c2: 4.4345e-04, d1: 9.9306e-02, d2: 0.0000e+00, s: 7.7289e-03, l1_reg: 3.2301e+02, l2_reg: 5.4504e+01\n",
      "2020/09/08, 22:00:55, Iteration: 3900, Train Loss: 1.8796e+01, c1: 4.0810e-02, c2: 5.6104e-04, d1: 9.1043e-02, d2: 0.0000e+00, s: 7.8813e-03, l1_reg: 3.2338e+02, l2_reg: 5.4965e+01\n",
      "2020/09/08, 22:00:56, Iteration: 4000, Train Loss: 1.4563e+01, c1: 2.3129e-02, c2: 4.1323e-04, d1: 8.1175e-02, d2: 0.0000e+00, s: 7.4500e-03, l1_reg: 3.2387e+02, l2_reg: 5.5493e+01\n",
      "2020/09/08, 22:00:57, Iteration: 4100, Train Loss: 1.4593e+01, c1: 2.5413e-02, c2: 4.9782e-04, d1: 7.0735e-02, d2: 0.0000e+00, s: 7.1865e-03, l1_reg: 3.2433e+02, l2_reg: 5.6029e+01\n",
      "2020/09/08, 22:00:58, Iteration: 4200, Train Loss: 1.7249e+01, c1: 2.4406e-02, c2: 8.4256e-04, d1: 6.3827e-02, d2: 0.0000e+00, s: 6.4254e-03, l1_reg: 3.2463e+02, l2_reg: 5.6442e+01\n",
      "2020/09/08, 22:00:58, Iteration: 4300, Train Loss: 1.6206e+01, c1: 4.7588e-02, c2: 5.4346e-04, d1: 6.0130e-02, d2: 0.0000e+00, s: 7.1071e-03, l1_reg: 3.2471e+02, l2_reg: 5.6714e+01\n",
      "2020/09/08, 22:00:59, Iteration: 4400, Train Loss: 1.3596e+01, c1: 2.5717e-02, c2: 5.2160e-04, d1: 5.8083e-02, d2: 0.0000e+00, s: 6.7197e-03, l1_reg: 3.2476e+02, l2_reg: 5.6936e+01\n",
      "2020/09/08, 22:01:00, Iteration: 4500, Train Loss: 1.4422e+01, c1: 4.2145e-02, c2: 4.5309e-04, d1: 5.6768e-02, d2: 0.0000e+00, s: 6.6868e-03, l1_reg: 3.2468e+02, l2_reg: 5.7078e+01\n",
      "2020/09/08, 22:01:01, Iteration: 4600, Train Loss: 1.3569e+01, c1: 2.0394e-02, c2: 6.0090e-04, d1: 5.5211e-02, d2: 0.0000e+00, s: 6.8059e-03, l1_reg: 3.2467e+02, l2_reg: 5.7250e+01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/08, 22:01:02, Iteration: 4700, Train Loss: 1.4563e+01, c1: 3.3174e-02, c2: 5.8209e-04, d1: 5.4242e-02, d2: 0.0000e+00, s: 6.6979e-03, l1_reg: 3.2458e+02, l2_reg: 5.7362e+01\n",
      "2020/09/08, 22:01:03, Iteration: 4800, Train Loss: 1.0249e+01, c1: 9.9031e-03, c2: 3.9142e-04, d1: 5.3442e-02, d2: 0.0000e+00, s: 6.2525e-03, l1_reg: 3.2443e+02, l2_reg: 5.7448e+01\n",
      "2020/09/08, 22:01:03, Iteration: 4900, Train Loss: 1.4232e+01, c1: 1.5695e-02, c2: 7.4536e-04, d1: 5.2086e-02, d2: 0.0000e+00, s: 6.9547e-03, l1_reg: 3.2450e+02, l2_reg: 5.7651e+01\n",
      "2020/09/08, 22:01:04, Iteration: 5000, Train Loss: 1.1172e+01, c1: 9.0653e-03, c2: 5.1527e-04, d1: 5.1125e-02, d2: 0.0000e+00, s: 6.5388e-03, l1_reg: 3.2446e+02, l2_reg: 5.7791e+01\n",
      "2020/09/08, 22:01:05, Iteration: 5100, Train Loss: 1.0927e+01, c1: 1.8649e-02, c2: 4.0126e-04, d1: 5.0496e-02, d2: 0.0000e+00, s: 6.6549e-03, l1_reg: 3.2431e+02, l2_reg: 5.7865e+01\n",
      "2020/09/08, 22:01:06, Iteration: 5200, Train Loss: 1.3467e+01, c1: 2.3539e-02, c2: 6.1559e-04, d1: 4.9571e-02, d2: 0.0000e+00, s: 6.5802e-03, l1_reg: 3.2422e+02, l2_reg: 5.8000e+01\n",
      "2020/09/08, 22:01:07, Iteration: 5300, Train Loss: 8.4773e+00, c1: 1.5120e-02, c2: 2.0968e-04, d1: 4.8685e-02, d2: 0.0000e+00, s: 6.6525e-03, l1_reg: 3.2413e+02, l2_reg: 5.8126e+01\n",
      "2020/09/08, 22:01:07, Iteration: 5400, Train Loss: 1.3779e+01, c1: 3.1641e-02, c2: 5.8062e-04, d1: 4.8083e-02, d2: 0.0000e+00, s: 6.8523e-03, l1_reg: 3.2409e+02, l2_reg: 5.8292e+01\n",
      "2020/09/08, 22:01:08, Iteration: 5500, Train Loss: 1.3066e+01, c1: 3.7084e-02, c2: 4.6604e-04, d1: 4.6970e-02, d2: 0.0000e+00, s: 6.7251e-03, l1_reg: 3.2404e+02, l2_reg: 5.8447e+01\n",
      "2020/09/08, 22:01:09, Iteration: 5600, Train Loss: 9.0668e+00, c1: 1.7848e-02, c2: 2.6860e-04, d1: 4.5960e-02, d2: 0.0000e+00, s: 6.6197e-03, l1_reg: 3.2397e+02, l2_reg: 5.8607e+01\n",
      "2020/09/08, 22:01:10, Iteration: 5700, Train Loss: 1.0617e+01, c1: 1.7937e-02, c2: 4.2819e-04, d1: 4.5410e-02, d2: 0.0000e+00, s: 5.9388e-03, l1_reg: 3.2379e+02, l2_reg: 5.8701e+01\n",
      "2020/09/08, 22:01:11, Iteration: 5800, Train Loss: 1.3369e+01, c1: 1.9701e-02, c2: 6.9897e-04, d1: 4.4092e-02, d2: 0.0000e+00, s: 5.9207e-03, l1_reg: 3.2386e+02, l2_reg: 5.8935e+01\n",
      "2020/09/08, 22:01:12, Iteration: 5900, Train Loss: 1.4345e+01, c1: 2.5468e-02, c2: 7.5004e-04, d1: 4.2981e-02, d2: 0.0000e+00, s: 6.0692e-03, l1_reg: 3.2381e+02, l2_reg: 5.9104e+01\n",
      "2020/09/08, 22:01:12, Iteration: 6000, Train Loss: 1.1329e+01, c1: 2.5513e-02, c2: 4.5834e-04, d1: 4.1944e-02, d2: 0.0000e+00, s: 5.6319e-03, l1_reg: 3.2373e+02, l2_reg: 5.9275e+01\n",
      "2020/09/08, 22:01:13, Iteration: 6100, Train Loss: 1.0526e+01, c1: 4.7548e-02, c2: 1.6763e-04, d1: 4.0944e-02, d2: 0.0000e+00, s: 6.7537e-03, l1_reg: 3.2365e+02, l2_reg: 5.9476e+01\n",
      "2020/09/08, 22:01:14, Iteration: 6200, Train Loss: 9.2151e+00, c1: 1.6463e-02, c2: 3.6088e-04, d1: 3.9599e-02, d2: 0.0000e+00, s: 5.7333e-03, l1_reg: 3.2371e+02, l2_reg: 5.9727e+01\n",
      "2020/09/08, 22:01:15, Iteration: 6300, Train Loss: 8.3855e+00, c1: 8.0247e-03, c2: 3.7196e-04, d1: 3.8634e-02, d2: 0.0000e+00, s: 6.0408e-03, l1_reg: 3.2370e+02, l2_reg: 5.9965e+01\n",
      "2020/09/08, 22:01:16, Iteration: 6400, Train Loss: 7.8581e+00, c1: 1.7286e-02, c2: 2.4160e-04, d1: 3.7136e-02, d2: 0.0000e+00, s: 6.1314e-03, l1_reg: 3.2364e+02, l2_reg: 6.0177e+01\n",
      "2020/09/08, 22:01:16, Iteration: 6500, Train Loss: 1.6373e+01, c1: 6.0516e-02, c2: 6.7244e-04, d1: 3.5967e-02, d2: 0.0000e+00, s: 6.2560e-03, l1_reg: 3.2365e+02, l2_reg: 6.0448e+01\n",
      "2020/09/08, 22:01:17, Iteration: 6600, Train Loss: 1.2351e+01, c1: 2.2306e-02, c2: 6.7056e-04, d1: 3.4143e-02, d2: 0.0000e+00, s: 5.8170e-03, l1_reg: 3.2369e+02, l2_reg: 6.0747e+01\n",
      "2020/09/08, 22:01:18, Iteration: 6700, Train Loss: 8.8257e+00, c1: 8.5019e-03, c2: 4.6896e-04, d1: 3.2859e-02, d2: 0.0000e+00, s: 5.7976e-03, l1_reg: 3.2368e+02, l2_reg: 6.1051e+01\n",
      "2020/09/08, 22:01:19, Iteration: 6800, Train Loss: 9.1665e+00, c1: 9.8108e-03, c2: 5.0104e-04, d1: 3.1750e-02, d2: 0.0000e+00, s: 5.4543e-03, l1_reg: 3.2360e+02, l2_reg: 6.1309e+01\n",
      "2020/09/08, 22:01:20, Iteration: 6900, Train Loss: 9.6244e+00, c1: 7.0729e-03, c2: 5.8761e-04, d1: 3.0410e-02, d2: 0.0000e+00, s: 4.7424e-03, l1_reg: 3.2360e+02, l2_reg: 6.1582e+01\n",
      "2020/09/08, 22:01:21, Iteration: 7000, Train Loss: 6.2313e+00, c1: 1.2391e-02, c2: 2.0908e-04, d1: 2.9013e-02, d2: 0.0000e+00, s: 5.2621e-03, l1_reg: 3.2354e+02, l2_reg: 6.1858e+01\n",
      "2020/09/08, 22:01:21, Iteration: 7100, Train Loss: 1.1537e+01, c1: 1.2021e-02, c2: 7.5328e-04, d1: 2.8017e-02, d2: 0.0000e+00, s: 4.8245e-03, l1_reg: 3.2349e+02, l2_reg: 6.2135e+01\n",
      "2020/09/08, 22:01:22, Iteration: 7200, Train Loss: 9.3631e+00, c1: 3.6338e-02, c2: 3.0366e-04, d1: 2.6927e-02, d2: 0.0000e+00, s: 5.4929e-03, l1_reg: 3.2336e+02, l2_reg: 6.2392e+01\n",
      "2020/09/08, 22:01:23, Iteration: 7300, Train Loss: 9.7109e+00, c1: 8.7564e-03, c2: 6.2322e-04, d1: 2.6030e-02, d2: 0.0000e+00, s: 5.0773e-03, l1_reg: 3.2340e+02, l2_reg: 6.2715e+01\n",
      "2020/09/08, 22:01:24, Iteration: 7400, Train Loss: 8.5564e+00, c1: 2.1609e-02, c2: 3.8550e-04, d1: 2.5405e-02, d2: 0.0000e+00, s: 4.3039e-03, l1_reg: 3.2328e+02, l2_reg: 6.2940e+01\n",
      "2020/09/08, 22:01:25, Iteration: 7500, Train Loss: 8.5991e+00, c1: 1.3811e-02, c2: 4.7415e-04, d1: 2.4766e-02, d2: 0.0000e+00, s: 4.4279e-03, l1_reg: 3.2331e+02, l2_reg: 6.3221e+01\n",
      "2020/09/08, 22:01:26, Iteration: 7600, Train Loss: 9.3941e+00, c1: 1.3984e-02, c2: 5.5674e-04, d1: 2.4283e-02, d2: 0.0000e+00, s: 4.4770e-03, l1_reg: 3.2326e+02, l2_reg: 6.3438e+01\n",
      "2020/09/08, 22:01:27, Iteration: 7700, Train Loss: 6.8906e+00, c1: 1.2870e-02, c2: 3.2192e-04, d1: 2.3844e-02, d2: 0.0000e+00, s: 4.2370e-03, l1_reg: 3.2325e+02, l2_reg: 6.3674e+01\n",
      "2020/09/08, 22:01:29, Iteration: 7800, Train Loss: 7.2428e+00, c1: 1.3056e-02, c2: 3.5943e-04, d1: 2.3429e-02, d2: 0.0000e+00, s: 4.7227e-03, l1_reg: 3.2325e+02, l2_reg: 6.3911e+01\n",
      "2020/09/08, 22:01:30, Iteration: 7900, Train Loss: 7.9172e+00, c1: 2.5697e-02, c2: 3.0435e-04, d1: 2.3040e-02, d2: 0.0000e+00, s: 3.8602e-03, l1_reg: 3.2306e+02, l2_reg: 6.4074e+01\n",
      "2020/09/08, 22:01:30, Iteration: 8000, Train Loss: 7.3615e+00, c1: 1.0122e-02, c2: 4.0893e-04, d1: 2.2600e-02, d2: 0.0000e+00, s: 4.1318e-03, l1_reg: 3.2296e+02, l2_reg: 6.4271e+01\n",
      "2020/09/08, 22:01:31, Iteration: 8100, Train Loss: 6.8198e+00, c1: 1.5676e-02, c2: 3.0254e-04, d1: 2.2267e-02, d2: 0.0000e+00, s: 3.9030e-03, l1_reg: 3.2292e+02, l2_reg: 6.4499e+01\n",
      "2020/09/08, 22:01:32, Iteration: 8200, Train Loss: 6.7799e+00, c1: 1.2734e-02, c2: 3.3067e-04, d1: 2.1997e-02, d2: 0.0000e+00, s: 3.8348e-03, l1_reg: 3.2281e+02, l2_reg: 6.4715e+01\n",
      "2020/09/08, 22:01:33, Iteration: 8300, Train Loss: 9.1704e+00, c1: 1.5532e-02, c2: 5.4536e-04, d1: 2.1636e-02, d2: 0.0000e+00, s: 3.6699e-03, l1_reg: 3.2276e+02, l2_reg: 6.4978e+01\n",
      "2020/09/08, 22:01:34, Iteration: 8400, Train Loss: 9.8932e+00, c1: 1.3157e-02, c2: 6.4467e-04, d1: 2.1308e-02, d2: 0.0000e+00, s: 3.4268e-03, l1_reg: 3.2261e+02, l2_reg: 6.5192e+01\n",
      "2020/09/08, 22:01:35, Iteration: 8500, Train Loss: 9.0202e+00, c1: 2.8208e-02, c2: 4.1019e-04, d1: 2.0975e-02, d2: 0.0000e+00, s: 4.0538e-03, l1_reg: 3.2243e+02, l2_reg: 6.5411e+01\n",
      "2020/09/08, 22:01:36, Iteration: 8600, Train Loss: 1.1455e+01, c1: 1.0998e-02, c2: 8.2986e-04, d1: 2.0566e-02, d2: 0.0000e+00, s: 4.0174e-03, l1_reg: 3.2242e+02, l2_reg: 6.5728e+01\n",
      "2020/09/08, 22:01:37, Iteration: 8700, Train Loss: 6.5040e+00, c1: 8.7194e-03, c2: 3.6028e-04, d1: 2.0292e-02, d2: 0.0000e+00, s: 3.3515e-03, l1_reg: 3.2218e+02, l2_reg: 6.5852e+01\n",
      "2020/09/08, 22:01:38, Iteration: 8800, Train Loss: 4.7245e+00, c1: 8.7221e-03, c2: 1.8424e-04, d1: 2.0099e-02, d2: 0.0000e+00, s: 2.9642e-03, l1_reg: 3.2195e+02, l2_reg: 6.5968e+01\n",
      "2020/09/08, 22:01:39, Iteration: 8900, Train Loss: 7.2827e+00, c1: 1.1684e-02, c2: 4.1206e-04, d1: 1.9938e-02, d2: 0.0000e+00, s: 3.3846e-03, l1_reg: 3.2153e+02, l2_reg: 6.6082e+01\n",
      "2020/09/08, 22:01:40, Iteration: 9000, Train Loss: 7.5102e+00, c1: 9.2679e-03, c2: 4.5703e-04, d1: 2.0131e-02, d2: 0.0000e+00, s: 3.2827e-03, l1_reg: 3.2139e+02, l2_reg: 6.6188e+01\n",
      "2020/09/08, 22:01:40, Iteration: 9100, Train Loss: 7.2089e+00, c1: 1.2639e-02, c2: 3.9517e-04, d1: 1.9933e-02, d2: 0.0000e+00, s: 2.6769e-03, l1_reg: 3.2127e+02, l2_reg: 6.6335e+01\n",
      "2020/09/08, 22:01:41, Iteration: 9200, Train Loss: 5.5176e+00, c1: 8.8153e-03, c2: 2.6498e-04, d1: 1.9863e-02, d2: 0.0000e+00, s: 2.7138e-03, l1_reg: 3.2103e+02, l2_reg: 6.6412e+01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/08, 22:01:42, Iteration: 9300, Train Loss: 6.1965e+00, c1: 1.1773e-02, c2: 3.0236e-04, d1: 1.9956e-02, d2: 0.0000e+00, s: 2.7956e-03, l1_reg: 3.2088e+02, l2_reg: 6.6478e+01\n",
      "2020/09/08, 22:01:43, Iteration: 9400, Train Loss: 8.8951e+00, c1: 5.3509e-03, c2: 6.3870e-04, d1: 1.9731e-02, d2: 0.0000e+00, s: 2.4272e-03, l1_reg: 3.2083e+02, l2_reg: 6.6583e+01\n",
      "2020/09/08, 22:01:44, Iteration: 9500, Train Loss: 7.3630e+00, c1: 4.1141e-03, c2: 4.9332e-04, d1: 2.0183e-02, d2: 0.0000e+00, s: 2.2886e-03, l1_reg: 3.2073e+02, l2_reg: 6.6663e+01\n",
      "2020/09/08, 22:01:45, Iteration: 9600, Train Loss: 7.4179e+00, c1: 1.7276e-02, c2: 3.7132e-04, d1: 1.9771e-02, d2: 0.0000e+00, s: 2.3561e-03, l1_reg: 3.2052e+02, l2_reg: 6.6628e+01\n",
      "2020/09/08, 22:01:46, Iteration: 9700, Train Loss: 6.3728e+00, c1: 6.3556e-03, c2: 3.7742e-04, d1: 1.9630e-02, d2: 0.0000e+00, s: 2.1534e-03, l1_reg: 3.2042e+02, l2_reg: 6.6630e+01\n",
      "2020/09/08, 22:01:47, Iteration: 9800, Train Loss: 6.9355e+00, c1: 2.0626e-02, c2: 2.8385e-04, d1: 2.0344e-02, d2: 0.0000e+00, s: 2.3343e-03, l1_reg: 3.2019e+02, l2_reg: 6.6680e+01\n",
      "2020/09/08, 22:01:48, Iteration: 9900, Train Loss: 5.2439e+00, c1: 9.6996e-03, c2: 2.3042e-04, d1: 1.9698e-02, d2: 0.0000e+00, s: 2.0892e-03, l1_reg: 3.1988e+02, l2_reg: 6.6541e+01\n",
      "2020/09/08, 22:01:49, Iteration: 10000, Train Loss: 5.9032e+00, c1: 9.8433e-03, c2: 2.9452e-04, d1: 1.9736e-02, d2: 0.0000e+00, s: 2.1497e-03, l1_reg: 3.1977e+02, l2_reg: 6.6491e+01\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.8/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "import jax\n",
    "import jax.numpy as jnp\n",
    "\n",
    "import os, sys\n",
    "sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(\"./\")))))\n",
    "\n",
    "from training import Agent\n",
    "from loss import params, loss_fn, evaluate_fn\n",
    "from data import generate_dataset, generate_batch_fn\n",
    "\n",
    "import config\n",
    "config.n_data = {\n",
    "    \"i\": 100,\n",
    "    \"b\": 100,\n",
    "    \"cx\": 501,\n",
    "    \"ct\": 201,\n",
    "    \"dx\": 100,\n",
    "    \"dt\": 600,\n",
    "}\n",
    "config.batch_size = {\n",
    "    \"dirichlet\": (config.n_data[\"i\"]+2*config.n_data[\"b\"]+config.n_data[\"dx\"]*config.n_data[\"dt\"]) // 10,\n",
    "    \"collocation\": (config.n_data[\"dx\"]*config.n_data[\"dt\"]+config.n_data[\"cx\"]*config.n_data[\"ct\"]) // 10,\n",
    "}\n",
    "config.iterations = 10000\n",
    "config.print_every = 100\n",
    "config.lr = 1e-3\n",
    "config.weights = {\n",
    "\t\"c1\": 1e2,\n",
    "\t\"c2\": 1e4,\n",
    "\t\"d1\": 1e2,\n",
    "\t\"d2\": 1.0,\n",
    "\t\"l1\": 1e-8,\n",
    "\t\"l2\": 1e-8,\n",
    "    \"s\": 1e-8\n",
    "}\n",
    "\n",
    "datasets = generate_dataset(config.n_data[\"i\"], config.n_data[\"b\"], config.n_data[\"cx\"], config.n_data[\"ct\"], config.n_data[\"dx\"], config.n_data[\"dt\"])\n",
    "batch_fn, evaluate_batch_fn = generate_batch_fn(config.key, config.batch_size, *datasets, config.weights)\n",
    "\n",
    "agent = Agent(params, loss_fn, evaluate_fn, \"models/{}\".format(config.NAME))\n",
    "agent.compile(config.optimizer, config.lr)\n",
    "agent.train(config.iterations, batch_fn, evaluate_batch_fn, config.print_every, config.save_every, config.loss_names, config.log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/08, 22:01:58, Iteration: 11000, Train Loss: 7.9659e+00, c1: 2.5234e-02, c2: 3.4725e-04, d1: 1.9700e-02, d2: 0.0000e+00, s: 1.6576e-03, l1_reg: 3.1657e+02, l2_reg: 6.6433e+01\n",
      "2020/09/08, 22:02:07, Iteration: 12000, Train Loss: 6.1335e+00, c1: 8.3440e-03, c2: 3.2912e-04, d1: 2.0079e-02, d2: 0.0000e+00, s: 1.4402e-03, l1_reg: 3.1561e+02, l2_reg: 6.6545e+01\n",
      "2020/09/08, 22:02:16, Iteration: 13000, Train Loss: 4.4874e+00, c1: 1.0187e-02, c2: 1.4917e-04, d1: 1.9770e-02, d2: 0.0000e+00, s: 1.4037e-03, l1_reg: 3.1347e+02, l2_reg: 6.6787e+01\n",
      "2020/09/08, 22:02:25, Iteration: 14000, Train Loss: 4.9124e+00, c1: 8.8751e-03, c2: 2.0697e-04, d1: 1.9553e-02, d2: 0.0000e+00, s: 1.1116e-03, l1_reg: 3.1104e+02, l2_reg: 6.7070e+01\n",
      "2020/09/08, 22:02:34, Iteration: 15000, Train Loss: 5.6853e+00, c1: 7.4402e-03, c2: 3.0228e-04, d1: 1.9184e-02, d2: 0.0000e+00, s: 1.1481e-03, l1_reg: 3.0828e+02, l2_reg: 6.8236e+01\n",
      "2020/09/08, 22:02:42, Iteration: 16000, Train Loss: 4.4842e+00, c1: 9.1985e-03, c2: 1.7132e-04, d1: 1.8512e-02, d2: 0.0000e+00, s: 1.1960e-03, l1_reg: 3.0649e+02, l2_reg: 6.9990e+01\n",
      "2020/09/08, 22:02:51, Iteration: 17000, Train Loss: 5.1008e+00, c1: 7.3536e-03, c2: 2.6076e-04, d1: 1.7579e-02, d2: 0.0000e+00, s: 7.4149e-04, l1_reg: 3.0293e+02, l2_reg: 7.2944e+01\n",
      "2020/09/08, 22:03:01, Iteration: 18000, Train Loss: 5.0448e+00, c1: 1.4138e-02, c2: 1.9391e-04, d1: 1.6919e-02, d2: 0.0000e+00, s: 8.1432e-04, l1_reg: 2.9940e+02, l2_reg: 7.7109e+01\n",
      "2020/09/08, 22:03:09, Iteration: 19000, Train Loss: 5.8571e+00, c1: 7.2173e-03, c2: 3.1353e-04, d1: 2.0000e-02, d2: 0.0000e+00, s: 6.8299e-04, l1_reg: 2.9554e+02, l2_reg: 8.0264e+01\n",
      "2020/09/08, 22:03:18, Iteration: 20000, Train Loss: 4.3965e+00, c1: 6.3894e-03, c2: 1.9813e-04, d1: 1.7763e-02, d2: 0.0000e+00, s: 4.5255e-04, l1_reg: 2.9209e+02, l2_reg: 8.1369e+01\n",
      "2020/09/08, 22:03:26, Iteration: 21000, Train Loss: 4.9106e+00, c1: 4.7898e-03, c2: 2.7642e-04, d1: 1.6673e-02, d2: 0.0000e+00, s: 3.7332e-04, l1_reg: 2.8546e+02, l2_reg: 8.1307e+01\n",
      "2020/09/08, 22:03:34, Iteration: 22000, Train Loss: 6.9524e+00, c1: 1.3888e-02, c2: 3.8733e-04, d1: 1.6903e-02, d2: 0.0000e+00, s: 2.6803e-04, l1_reg: 2.8175e+02, l2_reg: 8.1859e+01\n",
      "2020/09/08, 22:03:42, Iteration: 23000, Train Loss: 3.4821e+00, c1: 3.7666e-03, c2: 1.4435e-04, d1: 1.6619e-02, d2: 0.0000e+00, s: 1.5951e-04, l1_reg: 2.7666e+02, l2_reg: 8.2122e+01\n",
      "2020/09/08, 22:03:51, Iteration: 24000, Train Loss: 3.7111e+00, c1: 5.7620e-03, c2: 1.4542e-04, d1: 1.6807e-02, d2: 0.0000e+00, s: 3.3188e-04, l1_reg: 2.7086e+02, l2_reg: 8.2670e+01\n",
      "2020/09/08, 22:03:59, Iteration: 25000, Train Loss: 2.8853e+00, c1: 2.0129e-03, c2: 1.0052e-04, d1: 1.6788e-02, d2: 0.0000e+00, s: 5.9823e-04, l1_reg: 2.6505e+02, l2_reg: 8.2870e+01\n",
      "2020/09/08, 22:04:08, Iteration: 26000, Train Loss: 4.7838e+00, c1: 5.3049e-03, c2: 2.4533e-04, d1: 1.8000e-02, d2: 0.0000e+00, s: 6.4071e-04, l1_reg: 2.5775e+02, l2_reg: 8.4587e+01\n",
      "2020/09/08, 22:04:18, Iteration: 27000, Train Loss: 4.7884e+00, c1: 1.6083e-02, c2: 1.4773e-04, d1: 1.7028e-02, d2: 0.0000e+00, s: 5.5757e-04, l1_reg: 2.5105e+02, l2_reg: 8.5310e+01\n",
      "2020/09/08, 22:04:27, Iteration: 28000, Train Loss: 3.4864e+00, c1: 9.1276e-03, c2: 8.8888e-05, d1: 1.6848e-02, d2: 0.0000e+00, s: 3.5736e-04, l1_reg: 2.4492e+02, l2_reg: 8.6826e+01\n",
      "2020/09/08, 22:04:36, Iteration: 29000, Train Loss: 4.9542e+00, c1: 3.4339e-03, c2: 2.9439e-04, d1: 1.6669e-02, d2: 0.0000e+00, s: 2.4692e-04, l1_reg: 2.3969e+02, l2_reg: 8.8028e+01\n",
      "2020/09/08, 22:04:45, Iteration: 30000, Train Loss: 4.0027e+00, c1: 4.4773e-03, c2: 1.8262e-04, d1: 1.7288e-02, d2: 0.0000e+00, s: 9.2381e-05, l1_reg: 2.3194e+02, l2_reg: 8.9336e+01\n",
      "2020/09/08, 22:04:54, Iteration: 31000, Train Loss: 3.3716e+00, c1: 1.1238e-02, c2: 5.4231e-05, d1: 1.7055e-02, d2: 0.0000e+00, s: 5.6106e-05, l1_reg: 2.2793e+02, l2_reg: 9.0694e+01\n",
      "2020/09/08, 22:05:03, Iteration: 32000, Train Loss: 4.7319e+00, c1: 4.6566e-03, c2: 2.5755e-04, d1: 1.6908e-02, d2: 0.0000e+00, s: 2.2386e-05, l1_reg: 2.1930e+02, l2_reg: 9.1411e+01\n",
      "2020/09/08, 22:05:12, Iteration: 33000, Train Loss: 3.2675e+00, c1: 4.7521e-03, c2: 1.0826e-04, d1: 1.7097e-02, d2: 0.0000e+00, s: 3.0611e-06, l1_reg: 2.1352e+02, l2_reg: 9.1920e+01\n",
      "2020/09/08, 22:05:21, Iteration: 34000, Train Loss: 3.4181e+00, c1: 6.6998e-03, c2: 1.0845e-04, d1: 1.6636e-02, d2: 0.0000e+00, s: 8.6347e-07, l1_reg: 2.0848e+02, l2_reg: 9.2230e+01\n",
      "2020/09/08, 22:05:30, Iteration: 35000, Train Loss: 3.2069e+00, c1: 4.9067e-03, c2: 1.0307e-04, d1: 1.6856e-02, d2: 0.0000e+00, s: 1.3433e-08, l1_reg: 2.0427e+02, l2_reg: 9.2608e+01\n",
      "2020/09/08, 22:05:38, Iteration: 36000, Train Loss: 3.7559e+00, c1: 8.0788e-03, c2: 1.2504e-04, d1: 1.6976e-02, d2: 0.0000e+00, s: 5.5986e-09, l1_reg: 2.0179e+02, l2_reg: 9.2250e+01\n",
      "2020/09/08, 22:05:47, Iteration: 37000, Train Loss: 4.2252e+00, c1: 1.4929e-02, c2: 1.0037e-04, d1: 1.7286e-02, d2: 0.0000e+00, s: 1.4229e-07, l1_reg: 1.9861e+02, l2_reg: 9.2306e+01\n",
      "2020/09/08, 22:05:56, Iteration: 38000, Train Loss: 3.0323e+00, c1: 4.0677e-03, c2: 9.3270e-05, d1: 1.6928e-02, d2: 0.0000e+00, s: 2.1235e-08, l1_reg: 1.9580e+02, l2_reg: 9.2174e+01\n",
      "2020/09/08, 22:06:04, Iteration: 39000, Train Loss: 3.3698e+00, c1: 6.1953e-03, c2: 1.0714e-04, d1: 1.6788e-02, d2: 0.0000e+00, s: 5.6877e-07, l1_reg: 1.9242e+02, l2_reg: 9.2940e+01\n",
      "2020/09/08, 22:06:12, Iteration: 40000, Train Loss: 3.4830e+00, c1: 8.5373e-03, c2: 9.5522e-05, d1: 1.6741e-02, d2: 0.0000e+00, s: 9.2613e-09, l1_reg: 1.8969e+02, l2_reg: 9.2831e+01\n",
      "2020/09/08, 22:06:21, Iteration: 41000, Train Loss: 3.2422e+00, c1: 5.9876e-03, c2: 9.7304e-05, d1: 1.6704e-02, d2: 0.0000e+00, s: 2.8035e-09, l1_reg: 1.8752e+02, l2_reg: 9.2761e+01\n",
      "2020/09/08, 22:06:30, Iteration: 42000, Train Loss: 4.1699e+00, c1: 3.3164e-03, c2: 2.1319e-04, d1: 1.7064e-02, d2: 0.0000e+00, s: 7.4362e-07, l1_reg: 1.8556e+02, l2_reg: 9.2488e+01\n",
      "2020/09/08, 22:06:39, Iteration: 43000, Train Loss: 3.5088e+00, c1: 3.5746e-03, c2: 1.4931e-04, d1: 1.6582e-02, d2: 0.0000e+00, s: 1.8703e-08, l1_reg: 1.8414e+02, l2_reg: 9.2139e+01\n",
      "2020/09/08, 22:06:48, Iteration: 44000, Train Loss: 3.1590e+00, c1: 8.0938e-03, c2: 6.7777e-05, d1: 1.6718e-02, d2: 0.0000e+00, s: 4.5380e-08, l1_reg: 1.8213e+02, l2_reg: 9.2005e+01\n",
      "2020/09/08, 22:06:57, Iteration: 45000, Train Loss: 4.8196e+00, c1: 1.1170e-02, c2: 1.9868e-04, d1: 1.7159e-02, d2: 0.0000e+00, s: 2.9993e-07, l1_reg: 1.8029e+02, l2_reg: 9.2493e+01\n",
      "2020/09/08, 22:07:06, Iteration: 46000, Train Loss: 4.2064e+00, c1: 6.8174e-03, c2: 1.8198e-04, d1: 1.7049e-02, d2: 0.0000e+00, s: 8.5567e-07, l1_reg: 1.7642e+02, l2_reg: 9.1893e+01\n",
      "2020/09/08, 22:07:15, Iteration: 47000, Train Loss: 3.3784e+00, c1: 2.4577e-03, c2: 1.4689e-04, d1: 1.6638e-02, d2: 0.0000e+00, s: 1.2389e-06, l1_reg: 1.7589e+02, l2_reg: 9.1561e+01\n",
      "2020/09/08, 22:07:24, Iteration: 48000, Train Loss: 3.9505e+00, c1: 7.1180e-03, c2: 1.5772e-04, d1: 1.6615e-02, d2: 0.0000e+00, s: 1.8322e-08, l1_reg: 1.7406e+02, l2_reg: 9.1716e+01\n",
      "2020/09/08, 22:07:32, Iteration: 49000, Train Loss: 2.7721e+00, c1: 2.6208e-03, c2: 8.0611e-05, d1: 1.7039e-02, d2: 0.0000e+00, s: 2.2233e-08, l1_reg: 1.7160e+02, l2_reg: 9.1364e+01\n",
      "2020/09/08, 22:07:42, Iteration: 50000, Train Loss: 3.3510e+00, c1: 6.2624e-03, c2: 1.0237e-04, d1: 1.7011e-02, d2: 0.0000e+00, s: 7.6731e-08, l1_reg: 1.6895e+02, l2_reg: 9.1121e+01\n",
      "2020/09/08, 22:07:51, Iteration: 51000, Train Loss: 2.6700e+00, c1: 1.7907e-03, c2: 8.2471e-05, d1: 1.6663e-02, d2: 0.0000e+00, s: 1.3584e-08, l1_reg: 1.6669e+02, l2_reg: 9.1417e+01\n",
      "2020/09/08, 22:07:59, Iteration: 52000, Train Loss: 3.9761e+00, c1: 4.6125e-03, c2: 1.8379e-04, d1: 1.6770e-02, d2: 0.0000e+00, s: 2.1809e-10, l1_reg: 1.6331e+02, l2_reg: 9.1682e+01\n",
      "2020/09/08, 22:08:08, Iteration: 53000, Train Loss: 4.8637e+00, c1: 7.8076e-03, c2: 2.3722e-04, d1: 1.7107e-02, d2: 0.0000e+00, s: 5.1790e-07, l1_reg: 1.6259e+02, l2_reg: 9.1015e+01\n",
      "2020/09/08, 22:08:17, Iteration: 54000, Train Loss: 3.3880e+00, c1: 5.0914e-03, c2: 1.1979e-04, d1: 1.6809e-02, d2: 0.0000e+00, s: 6.3731e-10, l1_reg: 1.6077e+02, l2_reg: 9.1548e+01\n",
      "2020/09/08, 22:08:26, Iteration: 55000, Train Loss: 3.2575e+00, c1: 1.3217e-02, c2: 1.4067e-05, d1: 1.7951e-02, d2: 0.0000e+00, s: 7.0146e-11, l1_reg: 1.6084e+02, l2_reg: 9.1140e+01\n",
      "2020/09/08, 22:08:35, Iteration: 56000, Train Loss: 3.8042e+00, c1: 2.8541e-03, c2: 1.8305e-04, d1: 1.6883e-02, d2: 0.0000e+00, s: 1.8783e-07, l1_reg: 1.5802e+02, l2_reg: 9.0745e+01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/08, 22:08:44, Iteration: 57000, Train Loss: 3.6899e+00, c1: 7.1954e-03, c2: 1.2989e-04, d1: 1.6715e-02, d2: 0.0000e+00, s: 1.5156e-09, l1_reg: 1.5665e+02, l2_reg: 9.1268e+01\n",
      "2020/09/08, 22:08:53, Iteration: 58000, Train Loss: 3.3867e+00, c1: 1.6926e-03, c2: 1.5013e-04, d1: 1.7162e-02, d2: 0.0000e+00, s: 8.9961e-10, l1_reg: 1.5519e+02, l2_reg: 9.1112e+01\n",
      "2020/09/08, 22:09:02, Iteration: 59000, Train Loss: 3.0775e+00, c1: 3.0762e-03, c2: 1.0642e-04, d1: 1.7057e-02, d2: 0.0000e+00, s: 7.0333e-11, l1_reg: 1.5401e+02, l2_reg: 9.1587e+01\n",
      "2020/09/08, 22:09:11, Iteration: 60000, Train Loss: 3.4242e+00, c1: 5.6230e-03, c2: 1.2024e-04, d1: 1.6595e-02, d2: 0.0000e+00, s: 4.6859e-12, l1_reg: 1.5523e+02, l2_reg: 9.1084e+01\n",
      "2020/09/08, 22:09:20, Iteration: 61000, Train Loss: 4.1815e+00, c1: 5.6765e-04, c2: 2.4318e-04, d1: 1.6929e-02, d2: 0.0000e+00, s: 6.8296e-11, l1_reg: 1.5105e+02, l2_reg: 9.1660e+01\n",
      "2020/09/08, 22:09:29, Iteration: 62000, Train Loss: 3.4241e+00, c1: 3.3388e-03, c2: 1.4217e-04, d1: 1.6686e-02, d2: 0.0000e+00, s: 1.4445e-11, l1_reg: 1.4975e+02, l2_reg: 9.1151e+01\n",
      "2020/09/08, 22:09:38, Iteration: 63000, Train Loss: 2.8922e+00, c1: 7.4093e-04, c2: 1.1684e-04, d1: 1.6498e-02, d2: 0.0000e+00, s: 6.4036e-12, l1_reg: 1.4910e+02, l2_reg: 9.0931e+01\n",
      "2020/09/08, 22:09:47, Iteration: 64000, Train Loss: 2.7002e+00, c1: 5.9061e-03, c2: 3.8669e-05, d1: 1.7229e-02, d2: 0.0000e+00, s: 3.6865e-12, l1_reg: 1.4693e+02, l2_reg: 9.1520e+01\n",
      "2020/09/08, 22:09:55, Iteration: 65000, Train Loss: 3.5711e+00, c1: 4.5019e-03, c2: 1.4171e-04, d1: 1.7038e-02, d2: 0.0000e+00, s: 2.4251e-11, l1_reg: 1.4788e+02, l2_reg: 9.1126e+01\n",
      "2020/09/08, 22:10:04, Iteration: 66000, Train Loss: 2.8632e+00, c1: 2.0767e-03, c2: 9.8366e-05, d1: 1.6718e-02, d2: 0.0000e+00, s: 7.6682e-14, l1_reg: 1.4520e+02, l2_reg: 9.1102e+01\n",
      "2020/09/08, 22:10:13, Iteration: 67000, Train Loss: 2.8341e+00, c1: 2.5909e-03, c2: 9.0666e-05, d1: 1.6683e-02, d2: 0.0000e+00, s: 2.2411e-09, l1_reg: 1.4484e+02, l2_reg: 9.1175e+01\n",
      "2020/09/08, 22:10:22, Iteration: 68000, Train Loss: 3.4899e+00, c1: 3.1440e-03, c2: 1.5126e-04, d1: 1.6629e-02, d2: 0.0000e+00, s: 4.2702e-11, l1_reg: 1.4245e+02, l2_reg: 9.1184e+01\n",
      "2020/09/08, 22:10:31, Iteration: 69000, Train Loss: 4.3002e+00, c1: 8.9882e-03, c2: 1.7004e-04, d1: 1.7009e-02, d2: 0.0000e+00, s: 1.4545e-12, l1_reg: 1.4162e+02, l2_reg: 9.1460e+01\n",
      "2020/09/08, 22:10:40, Iteration: 70000, Train Loss: 3.3086e+00, c1: 3.3820e-03, c2: 1.3097e-04, d1: 1.6607e-02, d2: 0.0000e+00, s: 5.2061e-12, l1_reg: 1.3957e+02, l2_reg: 9.1433e+01\n",
      "2020/09/08, 22:10:49, Iteration: 71000, Train Loss: 4.1357e+00, c1: 4.1172e-03, c2: 2.0608e-04, d1: 1.6632e-02, d2: 0.0000e+00, s: 4.7627e-14, l1_reg: 1.3930e+02, l2_reg: 9.1727e+01\n",
      "2020/09/08, 22:10:58, Iteration: 72000, Train Loss: 3.8167e+00, c1: 1.5219e-02, c2: 6.1035e-05, d1: 1.6845e-02, d2: 0.0000e+00, s: 1.0609e-11, l1_reg: 1.3788e+02, l2_reg: 9.1865e+01\n",
      "2020/09/08, 22:11:07, Iteration: 73000, Train Loss: 3.3094e+00, c1: 3.3804e-03, c2: 1.2917e-04, d1: 1.6796e-02, d2: 0.0000e+00, s: 7.7607e-13, l1_reg: 1.3767e+02, l2_reg: 9.1521e+01\n",
      "2020/09/08, 22:11:16, Iteration: 74000, Train Loss: 3.3625e+00, c1: 2.8491e-03, c2: 1.4202e-04, d1: 1.6574e-02, d2: 0.0000e+00, s: 4.2511e-11, l1_reg: 1.3729e+02, l2_reg: 9.2112e+01\n",
      "2020/09/08, 22:11:25, Iteration: 75000, Train Loss: 2.9574e+00, c1: 6.4916e-03, c2: 6.3870e-05, d1: 1.6695e-02, d2: 0.0000e+00, s: 6.4251e-13, l1_reg: 1.3545e+02, l2_reg: 9.2565e+01\n",
      "2020/09/08, 22:11:34, Iteration: 76000, Train Loss: 4.2901e+00, c1: 1.0540e-02, c2: 1.5737e-04, d1: 1.6625e-02, d2: 0.0000e+00, s: 7.6213e-11, l1_reg: 1.3766e+02, l2_reg: 9.2658e+01\n",
      "2020/09/08, 22:11:43, Iteration: 77000, Train Loss: 3.9563e+00, c1: 3.7385e-03, c2: 1.9109e-04, d1: 1.6716e-02, d2: 0.0000e+00, s: 7.4933e-11, l1_reg: 1.3292e+02, l2_reg: 9.2412e+01\n",
      "2020/09/08, 22:11:52, Iteration: 78000, Train Loss: 2.6160e+00, c1: 2.2583e-03, c2: 7.1960e-05, d1: 1.6706e-02, d2: 0.0000e+00, s: 7.9751e-11, l1_reg: 1.3178e+02, l2_reg: 9.1756e+01\n",
      "2020/09/08, 22:12:01, Iteration: 79000, Train Loss: 3.2814e+00, c1: 6.1189e-03, c2: 1.0004e-04, d1: 1.6692e-02, d2: 0.0000e+00, s: 1.1157e-12, l1_reg: 1.3092e+02, l2_reg: 9.2837e+01\n",
      "2020/09/08, 22:12:09, Iteration: 80000, Train Loss: 2.6747e+00, c1: 2.6649e-03, c2: 7.0440e-05, d1: 1.7038e-02, d2: 0.0000e+00, s: 1.0527e-13, l1_reg: 1.2955e+02, l2_reg: 9.2463e+01\n",
      "2020/09/08, 22:12:18, Iteration: 81000, Train Loss: 3.0016e+00, c1: 1.6044e-03, c2: 1.1839e-04, d1: 1.6572e-02, d2: 0.0000e+00, s: 1.8042e-09, l1_reg: 1.2928e+02, l2_reg: 9.2896e+01\n",
      "2020/09/08, 22:12:27, Iteration: 82000, Train Loss: 2.4613e+00, c1: 3.7728e-03, c2: 4.1759e-05, d1: 1.6664e-02, d2: 0.0000e+00, s: 3.0388e-11, l1_reg: 1.2898e+02, l2_reg: 9.2356e+01\n",
      "2020/09/08, 22:12:36, Iteration: 83000, Train Loss: 2.8349e+00, c1: 2.9951e-03, c2: 8.8283e-05, d1: 1.6526e-02, d2: 0.0000e+00, s: 1.5122e-11, l1_reg: 1.2703e+02, l2_reg: 9.2594e+01\n",
      "2020/09/08, 22:12:45, Iteration: 84000, Train Loss: 3.7103e+00, c1: 5.1032e-03, c2: 1.5405e-04, d1: 1.6594e-02, d2: 0.0000e+00, s: 5.6291e-10, l1_reg: 1.2737e+02, l2_reg: 9.3154e+01\n",
      "2020/09/08, 22:12:54, Iteration: 85000, Train Loss: 3.0107e+00, c1: 8.7924e-04, c2: 1.2654e-04, d1: 1.6573e-02, d2: 0.0000e+00, s: 2.5322e-12, l1_reg: 1.2554e+02, l2_reg: 9.2732e+01\n",
      "2020/09/08, 22:13:03, Iteration: 86000, Train Loss: 3.1300e+00, c1: 6.1632e-03, c2: 8.5180e-05, d1: 1.6619e-02, d2: 0.0000e+00, s: 8.6858e-10, l1_reg: 1.2455e+02, l2_reg: 9.3475e+01\n",
      "2020/09/08, 22:13:12, Iteration: 87000, Train Loss: 3.8894e+00, c1: 4.0123e-03, c2: 1.8044e-04, d1: 1.6838e-02, d2: 0.0000e+00, s: 1.3604e-11, l1_reg: 1.2253e+02, l2_reg: 9.3552e+01\n",
      "2020/09/08, 22:13:21, Iteration: 88000, Train Loss: 3.7032e+00, c1: 7.3462e-03, c2: 1.3111e-04, d1: 1.6575e-02, d2: 0.0000e+00, s: 1.0163e-12, l1_reg: 1.2224e+02, l2_reg: 9.3632e+01\n",
      "2020/09/08, 22:13:29, Iteration: 89000, Train Loss: 3.4253e+00, c1: 1.5878e-02, c2: 1.6899e-05, d1: 1.6685e-02, d2: 0.0000e+00, s: 7.9806e-12, l1_reg: 1.2284e+02, l2_reg: 9.3666e+01\n",
      "2020/09/08, 22:13:38, Iteration: 90000, Train Loss: 2.6187e+00, c1: 4.5757e-04, c2: 9.1031e-05, d1: 1.6626e-02, d2: 0.0000e+00, s: 1.4174e-11, l1_reg: 1.2037e+02, l2_reg: 9.3973e+01\n",
      "2020/09/08, 22:13:47, Iteration: 91000, Train Loss: 2.2303e+00, c1: 8.2484e-04, c2: 4.9054e-05, d1: 1.6572e-02, d2: 0.0000e+00, s: 5.6492e-11, l1_reg: 1.2106e+02, l2_reg: 9.3526e+01\n",
      "2020/09/08, 22:13:56, Iteration: 92000, Train Loss: 3.3996e+00, c1: 2.7203e-03, c2: 1.4713e-04, d1: 1.6563e-02, d2: 0.0000e+00, s: 2.6000e-11, l1_reg: 1.2109e+02, l2_reg: 9.3947e+01\n",
      "2020/09/08, 22:14:05, Iteration: 93000, Train Loss: 6.3881e+00, c1: 2.2342e-03, c2: 4.4963e-04, d1: 1.6683e-02, d2: 0.0000e+00, s: 7.7077e-10, l1_reg: 1.2046e+02, l2_reg: 9.3449e+01\n",
      "2020/09/08, 22:14:14, Iteration: 94000, Train Loss: 3.9278e+00, c1: 4.0373e-03, c2: 1.8599e-04, d1: 1.6642e-02, d2: 0.0000e+00, s: 1.0775e-11, l1_reg: 1.1870e+02, l2_reg: 9.4270e+01\n",
      "2020/09/08, 22:14:23, Iteration: 95000, Train Loss: 5.0570e+00, c1: 7.6894e-03, c2: 2.6150e-04, d1: 1.6730e-02, d2: 0.0000e+00, s: 3.5956e-11, l1_reg: 1.2049e+02, l2_reg: 9.4502e+01\n",
      "2020/09/08, 22:14:32, Iteration: 96000, Train Loss: 3.8220e+00, c1: 1.6577e-03, c2: 2.0012e-04, d1: 1.6550e-02, d2: 0.0000e+00, s: 2.7030e-13, l1_reg: 1.1844e+02, l2_reg: 9.4607e+01\n",
      "2020/09/08, 22:14:41, Iteration: 97000, Train Loss: 2.3809e+00, c1: 9.7561e-04, c2: 6.2710e-05, d1: 1.6562e-02, d2: 0.0000e+00, s: 7.7164e-13, l1_reg: 1.1760e+02, l2_reg: 9.4241e+01\n",
      "2020/09/08, 22:14:50, Iteration: 98000, Train Loss: 2.5765e+00, c1: 3.1673e-03, c2: 6.0411e-05, d1: 1.6556e-02, d2: 0.0000e+00, s: 6.7614e-15, l1_reg: 1.1596e+02, l2_reg: 9.5255e+01\n",
      "2020/09/08, 22:14:59, Iteration: 99000, Train Loss: 3.5575e+00, c1: 3.9056e-03, c2: 1.4768e-04, d1: 1.6901e-02, d2: 0.0000e+00, s: 1.6956e-11, l1_reg: 1.1609e+02, l2_reg: 9.4748e+01\n",
      "2020/09/08, 22:15:07, Iteration: 100000, Train Loss: 3.6696e+00, c1: 3.3168e-03, c2: 1.6860e-04, d1: 1.6519e-02, d2: 0.0000e+00, s: 1.1719e-11, l1_reg: 1.1294e+02, l2_reg: 9.4818e+01\n",
      "2020/09/08, 22:15:16, Iteration: 101000, Train Loss: 3.1029e+00, c1: 1.2854e-03, c2: 1.3068e-04, d1: 1.6675e-02, d2: 0.0000e+00, s: 9.8342e-12, l1_reg: 1.1325e+02, l2_reg: 9.4809e+01\n",
      "2020/09/08, 22:15:25, Iteration: 102000, Train Loss: 4.2043e+00, c1: 3.3219e-03, c2: 2.2008e-04, d1: 1.6713e-02, d2: 0.0000e+00, s: 9.3893e-13, l1_reg: 1.1267e+02, l2_reg: 9.5412e+01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/08, 22:15:34, Iteration: 103000, Train Loss: 2.8723e+00, c1: 2.7905e-03, c2: 8.6029e-05, d1: 1.7330e-02, d2: 0.0000e+00, s: 2.2610e-15, l1_reg: 1.1156e+02, l2_reg: 9.6321e+01\n",
      "2020/09/08, 22:15:43, Iteration: 104000, Train Loss: 5.3564e+00, c1: 2.0635e-03, c2: 3.4922e-04, d1: 1.6579e-02, d2: 0.0000e+00, s: 1.2387e-12, l1_reg: 1.1251e+02, l2_reg: 9.6175e+01\n",
      "2020/09/08, 22:15:52, Iteration: 105000, Train Loss: 3.1264e+00, c1: 6.9147e-04, c2: 1.3738e-04, d1: 1.6835e-02, d2: 0.0000e+00, s: 5.4502e-09, l1_reg: 1.1462e+02, l2_reg: 9.5720e+01\n",
      "2020/09/08, 22:16:00, Iteration: 106000, Train Loss: 2.5717e+00, c1: 9.3354e-04, c2: 8.2470e-05, d1: 1.6537e-02, d2: 0.0000e+00, s: 1.1664e-11, l1_reg: 1.1158e+02, l2_reg: 9.6477e+01\n",
      "2020/09/08, 22:16:09, Iteration: 107000, Train Loss: 3.1812e+00, c1: 2.1122e-03, c2: 1.3207e-04, d1: 1.6493e-02, d2: 0.0000e+00, s: 4.1562e-10, l1_reg: 1.1033e+02, l2_reg: 9.6827e+01\n",
      "2020/09/08, 22:16:18, Iteration: 108000, Train Loss: 4.4728e+00, c1: 1.3664e-02, c2: 1.4095e-04, d1: 1.6968e-02, d2: 0.0000e+00, s: 8.7556e-12, l1_reg: 1.0954e+02, l2_reg: 9.6400e+01\n",
      "2020/09/08, 22:16:27, Iteration: 109000, Train Loss: 2.4580e+00, c1: 1.9834e-03, c2: 6.1076e-05, d1: 1.6489e-02, d2: 0.0000e+00, s: 1.3957e-09, l1_reg: 1.1038e+02, l2_reg: 9.7050e+01\n",
      "2020/09/08, 22:16:36, Iteration: 110000, Train Loss: 4.5565e+00, c1: 6.1755e-03, c2: 2.2279e-04, d1: 1.7110e-02, d2: 0.0000e+00, s: 1.4852e-11, l1_reg: 1.0947e+02, l2_reg: 9.6979e+01\n"
     ]
    }
   ],
   "source": [
    "config.iterations = 100000\n",
    "config.print_every = 1000\n",
    "agent.train(config.iterations, batch_fn, evaluate_batch_fn, config.print_every, config.save_every, config.loss_names, config.log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "/* Put everything inside the global mpl namespace */\n",
       "/* global mpl */\n",
       "window.mpl = {};\n",
       "\n",
       "mpl.get_websocket_type = function () {\n",
       "    if (typeof WebSocket !== 'undefined') {\n",
       "        return WebSocket;\n",
       "    } else if (typeof MozWebSocket !== 'undefined') {\n",
       "        return MozWebSocket;\n",
       "    } else {\n",
       "        alert(\n",
       "            'Your browser does not have WebSocket support. ' +\n",
       "                'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
       "                'Firefox 4 and 5 are also supported but you ' +\n",
       "                'have to enable WebSockets in about:config.'\n",
       "        );\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n",
       "    this.id = figure_id;\n",
       "\n",
       "    this.ws = websocket;\n",
       "\n",
       "    this.supports_binary = this.ws.binaryType !== undefined;\n",
       "\n",
       "    if (!this.supports_binary) {\n",
       "        var warnings = document.getElementById('mpl-warnings');\n",
       "        if (warnings) {\n",
       "            warnings.style.display = 'block';\n",
       "            warnings.textContent =\n",
       "                'This browser does not support binary websocket messages. ' +\n",
       "                'Performance may be slow.';\n",
       "        }\n",
       "    }\n",
       "\n",
       "    this.imageObj = new Image();\n",
       "\n",
       "    this.context = undefined;\n",
       "    this.message = undefined;\n",
       "    this.canvas = undefined;\n",
       "    this.rubberband_canvas = undefined;\n",
       "    this.rubberband_context = undefined;\n",
       "    this.format_dropdown = undefined;\n",
       "\n",
       "    this.image_mode = 'full';\n",
       "\n",
       "    this.root = document.createElement('div');\n",
       "    this.root.setAttribute('style', 'display: inline-block');\n",
       "    this._root_extra_style(this.root);\n",
       "\n",
       "    parent_element.appendChild(this.root);\n",
       "\n",
       "    this._init_header(this);\n",
       "    this._init_canvas(this);\n",
       "    this._init_toolbar(this);\n",
       "\n",
       "    var fig = this;\n",
       "\n",
       "    this.waiting = false;\n",
       "\n",
       "    this.ws.onopen = function () {\n",
       "        fig.send_message('supports_binary', { value: fig.supports_binary });\n",
       "        fig.send_message('send_image_mode', {});\n",
       "        if (mpl.ratio !== 1) {\n",
       "            fig.send_message('set_dpi_ratio', { dpi_ratio: mpl.ratio });\n",
       "        }\n",
       "        fig.send_message('refresh', {});\n",
       "    };\n",
       "\n",
       "    this.imageObj.onload = function () {\n",
       "        if (fig.image_mode === 'full') {\n",
       "            // Full images could contain transparency (where diff images\n",
       "            // almost always do), so we need to clear the canvas so that\n",
       "            // there is no ghosting.\n",
       "            fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
       "        }\n",
       "        fig.context.drawImage(fig.imageObj, 0, 0);\n",
       "    };\n",
       "\n",
       "    this.imageObj.onunload = function () {\n",
       "        fig.ws.close();\n",
       "    };\n",
       "\n",
       "    this.ws.onmessage = this._make_on_message_function(this);\n",
       "\n",
       "    this.ondownload = ondownload;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_header = function () {\n",
       "    var titlebar = document.createElement('div');\n",
       "    titlebar.classList =\n",
       "        'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n",
       "    var titletext = document.createElement('div');\n",
       "    titletext.classList = 'ui-dialog-title';\n",
       "    titletext.setAttribute(\n",
       "        'style',\n",
       "        'width: 100%; text-align: center; padding: 3px;'\n",
       "    );\n",
       "    titlebar.appendChild(titletext);\n",
       "    this.root.appendChild(titlebar);\n",
       "    this.header = titletext;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._init_canvas = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var canvas_div = (this.canvas_div = document.createElement('div'));\n",
       "    canvas_div.setAttribute(\n",
       "        'style',\n",
       "        'border: 1px solid #ddd;' +\n",
       "            'box-sizing: content-box;' +\n",
       "            'clear: both;' +\n",
       "            'min-height: 1px;' +\n",
       "            'min-width: 1px;' +\n",
       "            'outline: 0;' +\n",
       "            'overflow: hidden;' +\n",
       "            'position: relative;' +\n",
       "            'resize: both;'\n",
       "    );\n",
       "\n",
       "    function on_keyboard_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.key_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    canvas_div.addEventListener(\n",
       "        'keydown',\n",
       "        on_keyboard_event_closure('key_press')\n",
       "    );\n",
       "    canvas_div.addEventListener(\n",
       "        'keyup',\n",
       "        on_keyboard_event_closure('key_release')\n",
       "    );\n",
       "\n",
       "    this._canvas_extra_style(canvas_div);\n",
       "    this.root.appendChild(canvas_div);\n",
       "\n",
       "    var canvas = (this.canvas = document.createElement('canvas'));\n",
       "    canvas.classList.add('mpl-canvas');\n",
       "    canvas.setAttribute('style', 'box-sizing: content-box;');\n",
       "\n",
       "    this.context = canvas.getContext('2d');\n",
       "\n",
       "    var backingStore =\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        this.context.webkitBackingStorePixelRatio ||\n",
       "        this.context.mozBackingStorePixelRatio ||\n",
       "        this.context.msBackingStorePixelRatio ||\n",
       "        this.context.oBackingStorePixelRatio ||\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        1;\n",
       "\n",
       "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
       "\n",
       "    var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n",
       "        'canvas'\n",
       "    ));\n",
       "    rubberband_canvas.setAttribute(\n",
       "        'style',\n",
       "        'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n",
       "    );\n",
       "\n",
       "    var resizeObserver = new ResizeObserver(function (entries) {\n",
       "        var nentries = entries.length;\n",
       "        for (var i = 0; i < nentries; i++) {\n",
       "            var entry = entries[i];\n",
       "            var width, height;\n",
       "            if (entry.contentBoxSize) {\n",
       "                width = entry.contentBoxSize.inlineSize;\n",
       "                height = entry.contentBoxSize.blockSize;\n",
       "            } else {\n",
       "                width = entry.contentRect.width;\n",
       "                height = entry.contentRect.height;\n",
       "            }\n",
       "\n",
       "            // Keep the size of the canvas and rubber band canvas in sync with\n",
       "            // the canvas container.\n",
       "            canvas.setAttribute('width', width * mpl.ratio);\n",
       "            canvas.setAttribute('height', height * mpl.ratio);\n",
       "            canvas.setAttribute(\n",
       "                'style',\n",
       "                'width: ' + width + 'px; height: ' + height + 'px;'\n",
       "            );\n",
       "\n",
       "            rubberband_canvas.setAttribute('width', width);\n",
       "            rubberband_canvas.setAttribute('height', height);\n",
       "\n",
       "            // And update the size in Python. We ignore the initial 0/0 size\n",
       "            // that occurs as the element is placed into the DOM, which should\n",
       "            // otherwise not happen due to the minimum size styling.\n",
       "            if (width != 0 && height != 0) {\n",
       "                fig.request_resize(width, height);\n",
       "            }\n",
       "        }\n",
       "    });\n",
       "    resizeObserver.observe(canvas_div);\n",
       "\n",
       "    function on_mouse_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.mouse_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousedown',\n",
       "        on_mouse_event_closure('button_press')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseup',\n",
       "        on_mouse_event_closure('button_release')\n",
       "    );\n",
       "    // Throttle sequential mouse events to 1 every 20ms.\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousemove',\n",
       "        on_mouse_event_closure('motion_notify')\n",
       "    );\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseenter',\n",
       "        on_mouse_event_closure('figure_enter')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseleave',\n",
       "        on_mouse_event_closure('figure_leave')\n",
       "    );\n",
       "\n",
       "    canvas_div.addEventListener('wheel', function (event) {\n",
       "        if (event.deltaY < 0) {\n",
       "            event.step = 1;\n",
       "        } else {\n",
       "            event.step = -1;\n",
       "        }\n",
       "        on_mouse_event_closure('scroll')(event);\n",
       "    });\n",
       "\n",
       "    canvas_div.appendChild(canvas);\n",
       "    canvas_div.appendChild(rubberband_canvas);\n",
       "\n",
       "    this.rubberband_context = rubberband_canvas.getContext('2d');\n",
       "    this.rubberband_context.strokeStyle = '#000000';\n",
       "\n",
       "    this._resize_canvas = function (width, height, forward) {\n",
       "        if (forward) {\n",
       "            canvas_div.style.width = width + 'px';\n",
       "            canvas_div.style.height = height + 'px';\n",
       "        }\n",
       "    };\n",
       "\n",
       "    // Disable right mouse context menu.\n",
       "    this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n",
       "        return false;\n",
       "    });\n",
       "\n",
       "    function set_focus() {\n",
       "        canvas.focus();\n",
       "        canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    window.setTimeout(set_focus, 100);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'mpl-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'mpl-button-group';\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'mpl-button-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        var button = (fig.buttons[name] = document.createElement('button'));\n",
       "        button.classList = 'mpl-widget';\n",
       "        button.setAttribute('role', 'button');\n",
       "        button.setAttribute('aria-disabled', 'false');\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "\n",
       "        var icon_img = document.createElement('img');\n",
       "        icon_img.src = '_images/' + image + '.png';\n",
       "        icon_img.srcset = '_images/' + image + '_large.png 2x';\n",
       "        icon_img.alt = tooltip;\n",
       "        button.appendChild(icon_img);\n",
       "\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    var fmt_picker = document.createElement('select');\n",
       "    fmt_picker.classList = 'mpl-widget';\n",
       "    toolbar.appendChild(fmt_picker);\n",
       "    this.format_dropdown = fmt_picker;\n",
       "\n",
       "    for (var ind in mpl.extensions) {\n",
       "        var fmt = mpl.extensions[ind];\n",
       "        var option = document.createElement('option');\n",
       "        option.selected = fmt === mpl.default_extension;\n",
       "        option.innerHTML = fmt;\n",
       "        fmt_picker.appendChild(option);\n",
       "    }\n",
       "\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n",
       "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
       "    // which will in turn request a refresh of the image.\n",
       "    this.send_message('resize', { width: x_pixels, height: y_pixels });\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_message = function (type, properties) {\n",
       "    properties['type'] = type;\n",
       "    properties['figure_id'] = this.id;\n",
       "    this.ws.send(JSON.stringify(properties));\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_draw_message = function () {\n",
       "    if (!this.waiting) {\n",
       "        this.waiting = true;\n",
       "        this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    var format_dropdown = fig.format_dropdown;\n",
       "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
       "    fig.ondownload(fig, format);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_resize = function (fig, msg) {\n",
       "    var size = msg['size'];\n",
       "    if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n",
       "        fig._resize_canvas(size[0], size[1], msg['forward']);\n",
       "        fig.send_message('refresh', {});\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n",
       "    var x0 = msg['x0'] / mpl.ratio;\n",
       "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
       "    var x1 = msg['x1'] / mpl.ratio;\n",
       "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
       "    x0 = Math.floor(x0) + 0.5;\n",
       "    y0 = Math.floor(y0) + 0.5;\n",
       "    x1 = Math.floor(x1) + 0.5;\n",
       "    y1 = Math.floor(y1) + 0.5;\n",
       "    var min_x = Math.min(x0, x1);\n",
       "    var min_y = Math.min(y0, y1);\n",
       "    var width = Math.abs(x1 - x0);\n",
       "    var height = Math.abs(y1 - y0);\n",
       "\n",
       "    fig.rubberband_context.clearRect(\n",
       "        0,\n",
       "        0,\n",
       "        fig.canvas.width / mpl.ratio,\n",
       "        fig.canvas.height / mpl.ratio\n",
       "    );\n",
       "\n",
       "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n",
       "    // Updates the figure title.\n",
       "    fig.header.textContent = msg['label'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n",
       "    var cursor = msg['cursor'];\n",
       "    switch (cursor) {\n",
       "        case 0:\n",
       "            cursor = 'pointer';\n",
       "            break;\n",
       "        case 1:\n",
       "            cursor = 'default';\n",
       "            break;\n",
       "        case 2:\n",
       "            cursor = 'crosshair';\n",
       "            break;\n",
       "        case 3:\n",
       "            cursor = 'move';\n",
       "            break;\n",
       "    }\n",
       "    fig.rubberband_canvas.style.cursor = cursor;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_message = function (fig, msg) {\n",
       "    fig.message.textContent = msg['message'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n",
       "    // Request the server to send over a new figure.\n",
       "    fig.send_draw_message();\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n",
       "    fig.image_mode = msg['mode'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n",
       "    for (var key in msg) {\n",
       "        if (!(key in fig.buttons)) {\n",
       "            continue;\n",
       "        }\n",
       "        fig.buttons[key].disabled = !msg[key];\n",
       "        fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n",
       "    if (msg['mode'] === 'PAN') {\n",
       "        fig.buttons['Pan'].classList.add('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    } else if (msg['mode'] === 'ZOOM') {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.add('active');\n",
       "    } else {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Called whenever the canvas gets updated.\n",
       "    this.send_message('ack', {});\n",
       "};\n",
       "\n",
       "// A function to construct a web socket function for onmessage handling.\n",
       "// Called in the figure constructor.\n",
       "mpl.figure.prototype._make_on_message_function = function (fig) {\n",
       "    return function socket_on_message(evt) {\n",
       "        if (evt.data instanceof Blob) {\n",
       "            /* FIXME: We get \"Resource interpreted as Image but\n",
       "             * transferred with MIME type text/plain:\" errors on\n",
       "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
       "             * to be part of the websocket stream */\n",
       "            evt.data.type = 'image/png';\n",
       "\n",
       "            /* Free the memory for the previous frames */\n",
       "            if (fig.imageObj.src) {\n",
       "                (window.URL || window.webkitURL).revokeObjectURL(\n",
       "                    fig.imageObj.src\n",
       "                );\n",
       "            }\n",
       "\n",
       "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
       "                evt.data\n",
       "            );\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        } else if (\n",
       "            typeof evt.data === 'string' &&\n",
       "            evt.data.slice(0, 21) === 'data:image/png;base64'\n",
       "        ) {\n",
       "            fig.imageObj.src = evt.data;\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        var msg = JSON.parse(evt.data);\n",
       "        var msg_type = msg['type'];\n",
       "\n",
       "        // Call the  \"handle_{type}\" callback, which takes\n",
       "        // the figure and JSON message as its only arguments.\n",
       "        try {\n",
       "            var callback = fig['handle_' + msg_type];\n",
       "        } catch (e) {\n",
       "            console.log(\n",
       "                \"No handler for the '\" + msg_type + \"' message type: \",\n",
       "                msg\n",
       "            );\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        if (callback) {\n",
       "            try {\n",
       "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
       "                callback(fig, msg);\n",
       "            } catch (e) {\n",
       "                console.log(\n",
       "                    \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n",
       "                    e,\n",
       "                    e.stack,\n",
       "                    msg\n",
       "                );\n",
       "            }\n",
       "        }\n",
       "    };\n",
       "};\n",
       "\n",
       "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
       "mpl.findpos = function (e) {\n",
       "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
       "    var targ;\n",
       "    if (!e) {\n",
       "        e = window.event;\n",
       "    }\n",
       "    if (e.target) {\n",
       "        targ = e.target;\n",
       "    } else if (e.srcElement) {\n",
       "        targ = e.srcElement;\n",
       "    }\n",
       "    if (targ.nodeType === 3) {\n",
       "        // defeat Safari bug\n",
       "        targ = targ.parentNode;\n",
       "    }\n",
       "\n",
       "    // pageX,Y are the mouse positions relative to the document\n",
       "    var boundingRect = targ.getBoundingClientRect();\n",
       "    var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n",
       "    var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n",
       "\n",
       "    return { x: x, y: y };\n",
       "};\n",
       "\n",
       "/*\n",
       " * return a copy of an object with only non-object keys\n",
       " * we need this to avoid circular references\n",
       " * http://stackoverflow.com/a/24161582/3208463\n",
       " */\n",
       "function simpleKeys(original) {\n",
       "    return Object.keys(original).reduce(function (obj, key) {\n",
       "        if (typeof original[key] !== 'object') {\n",
       "            obj[key] = original[key];\n",
       "        }\n",
       "        return obj;\n",
       "    }, {});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.mouse_event = function (event, name) {\n",
       "    var canvas_pos = mpl.findpos(event);\n",
       "\n",
       "    if (name === 'button_press') {\n",
       "        this.canvas.focus();\n",
       "        this.canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    var x = canvas_pos.x * mpl.ratio;\n",
       "    var y = canvas_pos.y * mpl.ratio;\n",
       "\n",
       "    this.send_message(name, {\n",
       "        x: x,\n",
       "        y: y,\n",
       "        button: event.button,\n",
       "        step: event.step,\n",
       "        guiEvent: simpleKeys(event),\n",
       "    });\n",
       "\n",
       "    /* This prevents the web browser from automatically changing to\n",
       "     * the text insertion cursor when the button is pressed.  We want\n",
       "     * to control all of the cursor setting manually through the\n",
       "     * 'cursor' event from matplotlib */\n",
       "    event.preventDefault();\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n",
       "    // Handle any extra behaviour associated with a key event\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.key_event = function (event, name) {\n",
       "    // Prevent repeat events\n",
       "    if (name === 'key_press') {\n",
       "        if (event.which === this._key) {\n",
       "            return;\n",
       "        } else {\n",
       "            this._key = event.which;\n",
       "        }\n",
       "    }\n",
       "    if (name === 'key_release') {\n",
       "        this._key = null;\n",
       "    }\n",
       "\n",
       "    var value = '';\n",
       "    if (event.ctrlKey && event.which !== 17) {\n",
       "        value += 'ctrl+';\n",
       "    }\n",
       "    if (event.altKey && event.which !== 18) {\n",
       "        value += 'alt+';\n",
       "    }\n",
       "    if (event.shiftKey && event.which !== 16) {\n",
       "        value += 'shift+';\n",
       "    }\n",
       "\n",
       "    value += 'k';\n",
       "    value += event.which.toString();\n",
       "\n",
       "    this._key_event_extra(event, name);\n",
       "\n",
       "    this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n",
       "    if (name === 'download') {\n",
       "        this.handle_save(this, null);\n",
       "    } else {\n",
       "        this.send_message('toolbar_button', { name: name });\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n",
       "    this.message.textContent = tooltip;\n",
       "};\n",
       "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
       "\n",
       "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
       "\n",
       "mpl.default_extension = \"png\";/* global mpl */\n",
       "\n",
       "var comm_websocket_adapter = function (comm) {\n",
       "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
       "    // object with the appropriate methods. Currently this is a non binary\n",
       "    // socket, so there is still some room for performance tuning.\n",
       "    var ws = {};\n",
       "\n",
       "    ws.close = function () {\n",
       "        comm.close();\n",
       "    };\n",
       "    ws.send = function (m) {\n",
       "        //console.log('sending', m);\n",
       "        comm.send(m);\n",
       "    };\n",
       "    // Register the callback with on_msg.\n",
       "    comm.on_msg(function (msg) {\n",
       "        //console.log('receiving', msg['content']['data'], msg);\n",
       "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
       "        ws.onmessage(msg['content']['data']);\n",
       "    });\n",
       "    return ws;\n",
       "};\n",
       "\n",
       "mpl.mpl_figure_comm = function (comm, msg) {\n",
       "    // This is the function which gets called when the mpl process\n",
       "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
       "\n",
       "    var id = msg.content.data.id;\n",
       "    // Get hold of the div created by the display call when the Comm\n",
       "    // socket was opened in Python.\n",
       "    var element = document.getElementById(id);\n",
       "    var ws_proxy = comm_websocket_adapter(comm);\n",
       "\n",
       "    function ondownload(figure, _format) {\n",
       "        window.open(figure.canvas.toDataURL());\n",
       "    }\n",
       "\n",
       "    var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n",
       "\n",
       "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
       "    // web socket which is closed, not our websocket->open comm proxy.\n",
       "    ws_proxy.onopen();\n",
       "\n",
       "    fig.parent_element = element;\n",
       "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
       "    if (!fig.cell_info) {\n",
       "        console.error('Failed to find cell for figure', id, fig);\n",
       "        return;\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_close = function (fig, msg) {\n",
       "    var width = fig.canvas.width / mpl.ratio;\n",
       "    fig.root.removeEventListener('remove', this._remove_fig_handler);\n",
       "\n",
       "    // Update the output cell to use the data from the current canvas.\n",
       "    fig.push_to_output();\n",
       "    var dataURL = fig.canvas.toDataURL();\n",
       "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
       "    // the notebook keyboard shortcuts fail.\n",
       "    IPython.keyboard_manager.enable();\n",
       "    fig.parent_element.innerHTML =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "    fig.close_ws(fig, msg);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.close_ws = function (fig, msg) {\n",
       "    fig.send_message('closing', msg);\n",
       "    // fig.ws.close()\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n",
       "    // Turn the data on the canvas into data in the output cell.\n",
       "    var width = this.canvas.width / mpl.ratio;\n",
       "    var dataURL = this.canvas.toDataURL();\n",
       "    this.cell_info[1]['text/html'] =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Tell IPython that the notebook contents must change.\n",
       "    IPython.notebook.set_dirty(true);\n",
       "    this.send_message('ack', {});\n",
       "    var fig = this;\n",
       "    // Wait a second, then push the new image to the DOM so\n",
       "    // that it is saved nicely (might be nice to debounce this).\n",
       "    setTimeout(function () {\n",
       "        fig.push_to_output();\n",
       "    }, 1000);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'btn-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'btn-group';\n",
       "    var button;\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'btn-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        button = fig.buttons[name] = document.createElement('button');\n",
       "        button.classList = 'btn btn-default';\n",
       "        button.href = '#';\n",
       "        button.title = name;\n",
       "        button.innerHTML = '<i class=\"fa ' + image + ' fa-lg\"></i>';\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    // Add the status bar.\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message pull-right';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "\n",
       "    // Add the close button to the window.\n",
       "    var buttongrp = document.createElement('div');\n",
       "    buttongrp.classList = 'btn-group inline pull-right';\n",
       "    button = document.createElement('button');\n",
       "    button.classList = 'btn btn-mini btn-primary';\n",
       "    button.href = '#';\n",
       "    button.title = 'Stop Interaction';\n",
       "    button.innerHTML = '<i class=\"fa fa-power-off icon-remove icon-large\"></i>';\n",
       "    button.addEventListener('click', function (_evt) {\n",
       "        fig.handle_close(fig, {});\n",
       "    });\n",
       "    button.addEventListener(\n",
       "        'mouseover',\n",
       "        on_mouseover_closure('Stop Interaction')\n",
       "    );\n",
       "    buttongrp.appendChild(button);\n",
       "    var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n",
       "    titlebar.insertBefore(buttongrp, titlebar.firstChild);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._remove_fig_handler = function () {\n",
       "    this.close_ws(this, {});\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (el) {\n",
       "    el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n",
       "    el.addEventListener('remove', this._remove_fig_handler);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (el) {\n",
       "    // this is important to make the div 'focusable\n",
       "    el.setAttribute('tabindex', 0);\n",
       "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
       "    // off when our div gets focus\n",
       "\n",
       "    // location in version 3\n",
       "    if (IPython.notebook.keyboard_manager) {\n",
       "        IPython.notebook.keyboard_manager.register_events(el);\n",
       "    } else {\n",
       "        // location in version 2\n",
       "        IPython.keyboard_manager.register_events(el);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (event, _name) {\n",
       "    var manager = IPython.notebook.keyboard_manager;\n",
       "    if (!manager) {\n",
       "        manager = IPython.keyboard_manager;\n",
       "    }\n",
       "\n",
       "    // Check for shift+enter\n",
       "    if (event.shiftKey && event.which === 13) {\n",
       "        this.canvas_div.blur();\n",
       "        // select the cell after this one\n",
       "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
       "        IPython.notebook.select(index + 1);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    fig.ondownload(fig, null);\n",
       "};\n",
       "\n",
       "mpl.find_output_cell = function (html_output) {\n",
       "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
       "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
       "    // IPython event is triggered only after the cells have been serialised, which for\n",
       "    // our purposes (turning an active figure into a static one), is too late.\n",
       "    var cells = IPython.notebook.get_cells();\n",
       "    var ncells = cells.length;\n",
       "    for (var i = 0; i < ncells; i++) {\n",
       "        var cell = cells[i];\n",
       "        if (cell.cell_type === 'code') {\n",
       "            for (var j = 0; j < cell.output_area.outputs.length; j++) {\n",
       "                var data = cell.output_area.outputs[j];\n",
       "                if (data.data) {\n",
       "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
       "                    data = data.data;\n",
       "                }\n",
       "                if (data['text/html'] === html_output) {\n",
       "                    return [cell, data, j];\n",
       "                }\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "};\n",
       "\n",
       "// Register the function which deals with the matplotlib target/channel.\n",
       "// The kernel may be null if the page has been refreshed.\n",
       "if (IPython.notebook.kernel !== null) {\n",
       "    IPython.notebook.kernel.comm_manager.register_target(\n",
       "        'matplotlib',\n",
       "        mpl.mpl_figure_comm\n",
       "    );\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<img src=\"\" width=\"1342.2333333333333\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from loss import inverse_model, direct_model\n",
    "from data import domain\n",
    "import jax.numpy as jnp\n",
    "import numpy as np\n",
    "\n",
    "inverse_params = agent.params[1]\n",
    "x_test = jnp.linspace(*domain[:, 0], 100).reshape((-1, 1))\n",
    "a_fn = lambda x: 1+0.5*x**2\n",
    "a_pred = inverse_model(inverse_params, x_test)\n",
    "a_true = a_fn(x_test)\n",
    "\n",
    "direct_params = agent.params[0]\n",
    "t_test = domain[1, 1]*jnp.ones_like(x_test)\n",
    "uv_pred = direct_model(direct_params, jnp.hstack([x_test, t_test]))\n",
    "\n",
    "from scipy.io import loadmat\n",
    "data_true = loadmat(\"problem2_3_snapshot_epsilon_1e-12.mat\")\n",
    "u_true, v_true = data_true[\"u_snapshots\"][:, -1], data_true[\"v_snapshots\"][:, -1]\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib notebook\n",
    "\n",
    "f, ax = plt.subplots(1, 3, figsize = (15, 5))\n",
    "ax[0].plot(x_test, a_pred, label = \"pred\")\n",
    "ax[0].plot(x_test, a_true, label = \"true\")\n",
    "ax[0].set_title(\"a\")\n",
    "ax[1].plot(x_test, uv_pred[:, 0:1], label = \"pred\")\n",
    "ax[1].plot(x_test, u_true, label = \"true\")\n",
    "ax[1].set_title(\"u\")\n",
    "ax[2].plot(x_test, uv_pred[:, 1:2], label = \"pred\")\n",
    "ax[2].plot(x_test, v_true, label = \"true\")\n",
    "ax[2].set_title(\"v\")\n",
    "for ax_ in ax:\n",
    "\tax_.legend()\n",
    "\tax_.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
